Load required packages
# load pacman package to load or install other requried libraries
if (!require('pacman')) install.packages('pacman'); library(pacman)
# load (install if required) packages from CRAN
p_load("here","tidyverse","lubridate","janitor","data.table","plotly","doParallel")
# load/install packages from GitHub
p_load_gh("reichlab/zoltr", "reichlab/covidHubUtils")
Utilizing zoltr package to enable access to Zoltar API to the forecast archive and covidHubUtils package to that provide functions to read, plot and score forecast data.
Â
Importing observed data
We will use the Covid-19 cases as updated by CDC. We will download data from the data Table for daily Case Trends for The United States found here.
Also, we will pull daily COVID-19 cases from CDC at a state-level found here.
Data extracted September, 06, 2021. Here is the top rows of the incident COVID-19 cases data.
# csv export of daily cases from CDC COVID data tracker at national level
national_cases_daily <- read_csv(here("data", "data_table_for_daily_case_trends__the_united_states.csv"),
col_types = cols(Date = col_date(format = "%b %d %Y")),
skip = 2)
# csv export of daily cases from CDC COVID data tracker at state level
states_cases_daily <- read_csv(here("data", "United_States_COVID-19_Cases_and_Deaths_by_State_over_Time.csv"),
col_types = cols(submission_date = col_date(format = "%m/%d/%Y")))
head(national_cases_daily)
Â
Preparing data
We’ll aggregate the daily case counts into weeks, starting on Sunday as is the same as an epidemiological week (referred to as MMWR week, more info found here)
# clean up column names
national_cases_daily <- national_cases_daily %>%
clean_names()
# aggregate daily COVID cases into weekly case counts,
# start week on Sunday according to epidemiological week (MMWR week)
week <- as.Date(cut(national_cases_daily$date, "week", start.on.monday = FALSE))
national_cases_weekly <- aggregate(new_cases ~ week, national_cases_daily, sum)
head(national_cases_weekly)
# include last day of week and filter dates after August
national_cases_weekly <- national_cases_weekly %>%
mutate(week_end = week + days(6)) %>%
filter(week < "2021-09-01")
The forecast models are pulled from the Zoltar forecast model archive using the covidHubUtils package.
# check which US-specific models are contained in the forecast archive.
model_names <- get_all_models(hub = "US")
There are 106 models within the Zoltar forecast archive from the U.S COVID-19 Forecasts Hub.
Load the incident cases forecasts models of 1 through 4 week horizons of a select number of US-specific models that are published by the CDC and are contained within the Zoltar forecast archive.
if (bypass_load_from_zoltar == 0) {
# load forecasts of all models
system.time(all_inc_case_targets <- load_forecasts(
location = "US",
hub = "US",
types = c("point","quantile"),
targets = paste(1:4, "wk ahead inc case"),
as_of = "2021-09-06",
source = "zoltar"))
# write model forecast to directory
write_csv(all_inc_case_targets, here("data", "all_inc_case_targets.csv"))
} else {
# read pre-queried forecast data
all_inc_case_targets <- read_csv(here("data", "all_inc_case_targets.csv"))
}
# filter for a select few point forecasts that were submitted in from zoltar forecast archive
inc_case_targets <- all_inc_case_targets %>%
filter(model %in% c("COVIDhub-ensemble", "CovidAnalytics-DELPHI", "CU-select",
"IHME-CurveFit", "LANL-GrowthRate","USC-SI_kJalpha",
"JHU_IDD-CovidSP", "UVA-Ensemble"))
52 models stored in the Zoltar forecast archive have submitted a weekly incident case forecast.
# display top rows of forecast data
head(inc_case_targets)
inc_case_targets <- inc_case_targets %>%
mutate(forecast_day = lubridate::wday(forecast_date, label = TRUE, abbr = FALSE)) %>%
select(model, forecast_date, forecast_day, everything())
Filter the models for respective 1 through 4 week horizon point forecasts.
# filter for 1-4 week horizon point forecasts
for (wk in 1:4) {
wk_forecasts <- adj_inc_case_targets %>%
filter(horizon == wk,
type == "point") %>%
select(-quantile) %>%
dplyr::distinct(model, adj_forecast_date, .keep_all = TRUE)
assign(paste0("inc_case_targets_",wk,"_week"), wk_forecasts)
}
Visualizing forecast data
Plot the CDC actual COVID-19 cases versus the forecast predictions for the select models.
# plot the weekly cases for United States according to CDC data as of 09/06/2021
p <- ggplot(data = national_cases_weekly, aes(x = week_end, y = new_cases)) +
geom_point() +
geom_line() +
labs(title = "CDC weekly incident COVID-19 cases")
p

# combine plots
p + geom_line(data = inc_case_targets_4_week, aes(x = target_end_date, y = value, color = model)) +
geom_point(data = inc_case_targets_4_week, aes(x = target_end_date, y = value, color = model)) +
labs(title = "CDC weekly incident COVID-19 cases versus various model point forecasts") +
theme(legend.position = "bottom")

Interactive plot of CDC observed cases versus 4 week horizon forecasts
# plot 1 week horizon point forecast
plot_inc_forecast(horizon = 1)
# plot 4 week horizon point foreacst
plot_inc_forecast(horizon = 4)
IHME-Curvefit only predicted incident COVID-cases for a few weeks before focusing on predicting hospitalizations and deaths, according to predictions stored in Zoltar forecast archive.
Identifying peaks of COVID-19 cases
# a 'peak' is defined as a local maxima with m points either side of it being smaller than it. hence, the bigger the parameter m, the more stringent is the peak finding procedure
# https://github.com/stas-g/findPeaks
find_peaks <- function (x, m = 3){
shape <- diff(sign(diff(x, na.pad = FALSE)))
pks <- sapply(which(shape < 0), FUN = function(i){
z <- i - m + 1
z <- ifelse(z > 0, z, 1)
w <- i + m + 1
w <- ifelse(w < length(x), w, length(x))
if(all(x[c(z : i, (i + 2) : w)] <= x[i + 1])) return(i + 1) else return(numeric(0))
})
pks <- unlist(pks)
pks
}
Peaks of observed weekly incident COVID-19 cases based on CDC data
# find peaks of observed cases
peak_cases <- national_cases_weekly[find_peaks(national_cases_weekly$new_cases, m = 3),]
# plot peaks for observed covid cases
q <- p + geom_point(data = peak_cases, aes(x = week_end, y = new_cases, color = "CDC actual")) +
labs(title = "Observed peaks of weekly incident COVID-19 cases",
x = "Weeks", y = "Incident cases", color = "Peaks") +
theme(legend.position = "bottom")
q

# get peak values of all values
model_peaks_list <- list()
# get peaks for 4 week horizon forecast
for (i in unique(inc_case_targets_4_week$model)) {
models <- inc_case_targets_4_week %>%
filter(model == i)
peaks <- models[find_peaks(models$value, m = 3),]
model_peaks_list[[i]] <- peaks
}
# combine the df of peaks
model_peaks <- bind_rows(model_peaks_list)
Look at the peaks associated with forecast models
# combine plots of cdc actual peaks with forecast model peaks
q + geom_point(data = model_peaks, aes(x = target_end_date, y = value, color = model))

# find magnitude and temporal differences in the peaks
cdc_peaks <- peak_cases %>%
mutate(model = rep("CDC", nrow(peak_cases))) %>%
select(week_end, model, new_cases) %>%
rename("peaks" = "new_cases")
model_observed_pks <- model_peaks %>%
select(target_end_date, model, value) %>%
rename("week_end" = "target_end_date",
"peaks" = "value") %>%
bind_rows(cdc_peaks)
Score each model weekly
To score models using covidHubUtils, data frame needs to be in same format as one gotten from load_truth function.
# load truth data frame from file
bypass_truth_zoltar <- 1
# load a truth data frame from covidHub
if (bypass_truth_zoltar == 0) {
jhu_truth_df <- load_truth(
truth_source = c("JHU"),
target_variable = c("inc case"),
truth_end_date = "2021-09-04",
temporal_resolution = "weekly",
hub = "US",
locations = "US")
# write truth data frame to csv file
write_csv(jhu_truth_df, here("data", "jhu_truth_df.csv"))
} else {
# load truth df from file
jhu_truth_df <- read_csv(here("data", "jhu_truth_df.csv"))
}
# top rows of truth dataframe
head(jhu_truth_df)
WIS scores for each model and different forecast horizons
# get each models weekly wis score for each horizon
for (i in 1:4) {
wis_scores <- joint_df %>%
filter(horizon == i) %>%
# group_by(model, target_end_date) %>%
select(model, horizon, abs_error, wis, forecast_value, true_value, target_end_date, forecast_date) %>%
arrange(desc(wis))
assign(paste0("wis_scores_",i), wis_scores)
}
wis_scores_1
wis_scores_2
wis_scores_3
wis_scores_4
Median absolute percent error (mape) for each model and forecast horizon
# calculate the median absolute percent error for each model
joint_df %>%
group_by(model, horizon) %>%
summarise(num_of_forecasts = n(),
mape = median(abs((true_value-forecast_value)/true_value))*100) %>%
arrange(desc(mape))
# filter out dates before July 2020, labor day, thanksgiving, Christmas and New Years
national_cases_weekly %>%
filter(week_end > "2020-07-04" &
!week_end == "2020-09-12" &
!week_end == "2020-12-26" &
!week_end == "2021-01-02")
# check the weeks of monotonic increasing and decreasing cases time periods
weekly_cases <- data.table(national_cases_weekly)
weekly_cases <- weekly_cases %>%
mutate(diff = new_cases - lag(new_cases, 1, 0),
inc = if_else(diff>0, 1, 0),
dec = if_else(diff<0, 1, 0),
inc_dec_count = sequence(rle(as.character(inc))$lengths))
# consecutive weeks of increasing COVID cases
weekly_cases[which(inc==1), ]
# consecutive weeks of increasing COVID cases
weekly_cases[which(inc==0), ]
# check the weeks of monotonic increasing and decreasing cases time periods for each model
models_inc_dec <- all_inc_case_targets %>%
filter(type == "point",
horizon == 4) %>%
select(-location, -temporal_resolution, -target_variable, -type, -quantile, -location_name,
-population, -starts_with("geo"), -abbreviation, -full_location_name) %>%
group_by(model) %>%
mutate(diff = value - lag(value, 1,0),
inc = if_else(diff>0, 1, 0),
dec = if_else(diff<0, 1, 0),
inc_dec_count = sequence(rle(as.character(inc))$lengths))
models_inc_dec
# check the weeks of monotonic increasing and decreasing cases time periods for each model
models_horizon_inc_dec <- all_inc_case_targets %>%
filter(type == "point") %>%
select(-location, -temporal_resolution, -target_variable, -type, -quantile, -location_name,
-population, -starts_with("geo"), -abbreviation, -full_location_name) %>%
group_by(model, horizon) %>%
mutate(diff = value - lag(value, 1,0),
inc = if_else(diff>0, 1, 0),
dec = if_else(diff<0, 1, 0),
inc_dec_count = sequence(rle(as.character(inc))$lengths))
models_horizon_inc_dec
LS0tCnRpdGxlOiAiQ292aWQtMTkgZm9yZWNhc3QgbW9kZWwgZXhwbG9yYXRpb24iCnN1YnRpdGxlOiAiRXhwbG9yaW5nIGFuZCB2aXN1YWxpemluZyBDT1ZJRC0xOSBmb3JlY2FzdCBtb2RlbHMiCmRhdGU6ICJMYXN0IHVwZGF0ZWQ6IGByIGZvcm1hdChTeXMudGltZSgpLCclQiAlZCwgJVknKWAiCm91dHB1dDogCiAgaHRtbF9kb2N1bWVudDoKICAgIHRvYzogZmFsc2UKICAgIHRvY19mbG9hdDogZmFsc2UKICAgIGRmX3ByaW50OiBwYWdlZAogICAgY29kZV9kb3dubG9hZDogdHJ1ZQogICAgY29kZV9mb2xkaW5nOiBoaWRlCmVkaXRvcl9vcHRpb25zOiAKICBjaHVua19vdXRwdXRfdHlwZTogY29uc29sZQotLS0KCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFfQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUsIGZpZy5hbGlnbiA9ICJjZW50ZXIiLCBmaWcud2lkdGg9MTIsIGZpZy5oZWlnaHQ9NikKYGBgCgpgYGB7ciwgZWNobz1GQUxTRSwgaW5jbHVkZT1GQUxTRX0KIyBkaXNhYmxlIHNjaWVudGlmaWMgbm90YXRpb24Kb3B0aW9ucyhzY2lwZW4gPSA5OTksIGRpZ2l0cyA9IDIpCgojIGNsZWFyIGVudmlyb25tZW50CnJtKGxpc3QgPSBscygpKSAgIyBjbGVhciBtZW1vcnkKYGBgCgpMb2FkIHJlcXVpcmVkIHBhY2thZ2VzCmBgYHtyLCBtZXNzYWdlPUZBTFNFfQojIGxvYWQgcGFjbWFuIHBhY2thZ2UgdG8gbG9hZCBvciBpbnN0YWxsIG90aGVyIHJlcXVyaWVkIGxpYnJhcmllcwppZiAoIXJlcXVpcmUoJ3BhY21hbicpKSBpbnN0YWxsLnBhY2thZ2VzKCdwYWNtYW4nKTsgbGlicmFyeShwYWNtYW4pCgojIGxvYWQgKGluc3RhbGwgaWYgcmVxdWlyZWQpIHBhY2thZ2VzIGZyb20gQ1JBTgpwX2xvYWQoImhlcmUiLCJ0aWR5dmVyc2UiLCJsdWJyaWRhdGUiLCJqYW5pdG9yIiwiZGF0YS50YWJsZSIsInBsb3RseSIsImRvUGFyYWxsZWwiKQoKIyBsb2FkL2luc3RhbGwgcGFja2FnZXMgZnJvbSBHaXRIdWIKcF9sb2FkX2doKCJyZWljaGxhYi96b2x0ciIsICJyZWljaGxhYi9jb3ZpZEh1YlV0aWxzIikKYGBgCgpVdGlsaXppbmcgYHpvbHRyYCBwYWNrYWdlIHRvIGVuYWJsZSBhY2Nlc3MgdG8gWm9sdGFyIEFQSSB0byB0aGUgZm9yZWNhc3QgYXJjaGl2ZSBhbmQgYGNvdmlkSHViVXRpbHNgIHBhY2thZ2UgdG8gdGhhdCBwcm92aWRlIGZ1bmN0aW9ucyB0byByZWFkLCBwbG90IGFuZCBzY29yZSBmb3JlY2FzdCBkYXRhLgoKXCAgCgojIyMgSW1wb3J0aW5nIG9ic2VydmVkIGRhdGEKV2Ugd2lsbCB1c2UgdGhlIENvdmlkLTE5IGNhc2VzIGFzIHVwZGF0ZWQgYnkgQ0RDLiBXZSB3aWxsIGRvd25sb2FkIGRhdGEgZnJvbSB0aGUgZGF0YSBUYWJsZSBmb3IgZGFpbHkgQ2FzZSBUcmVuZHMgZm9yIFRoZSBVbml0ZWQgU3RhdGVzIFtmb3VuZCBoZXJlXSggaHR0cHM6Ly9jb3ZpZC5jZGMuZ292L2NvdmlkLWRhdGEtdHJhY2tlci8jdHJlbmRzX2RhaWx5Y2FzZXMpLiAKCkFsc28sIHdlIHdpbGwgcHVsbCBkYWlseSBDT1ZJRC0xOSBjYXNlcyBmcm9tIENEQyBhdCBhIHN0YXRlLWxldmVsIFtmb3VuZCBoZXJlXShodHRwczovL2RhdGEuY2RjLmdvdi9DYXNlLVN1cnZlaWxsYW5jZS9Vbml0ZWQtU3RhdGVzLUNPVklELTE5LUNhc2VzLWFuZC1EZWF0aHMtYnktU3RhdGUtby85bWZxLWNiMzYvZGF0YSkuCgpEYXRhIGV4dHJhY3RlZCBTZXB0ZW1iZXIsIDA2LCAyMDIxLiBIZXJlIGlzIHRoZSB0b3Agcm93cyBvZiB0aGUgaW5jaWRlbnQgQ09WSUQtMTkgY2FzZXMgZGF0YS4KYGBge3IsIG1lc3NhZ2U9RkFMU0V9CiMgY3N2IGV4cG9ydCBvZiBkYWlseSBjYXNlcyBmcm9tIENEQyBDT1ZJRCBkYXRhIHRyYWNrZXIgYXQgbmF0aW9uYWwgbGV2ZWwKbmF0aW9uYWxfY2FzZXNfZGFpbHkgPC0gcmVhZF9jc3YoaGVyZSgiZGF0YSIsICJkYXRhX3RhYmxlX2Zvcl9kYWlseV9jYXNlX3RyZW5kc19fdGhlX3VuaXRlZF9zdGF0ZXMuY3N2IiksIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2xfdHlwZXMgPSBjb2xzKERhdGUgPSBjb2xfZGF0ZShmb3JtYXQgPSAiJWIgJWQgJVkiKSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNraXAgPSAyKQoKCiMgY3N2IGV4cG9ydCBvZiBkYWlseSBjYXNlcyBmcm9tIENEQyBDT1ZJRCBkYXRhIHRyYWNrZXIgYXQgc3RhdGUgbGV2ZWwKc3RhdGVzX2Nhc2VzX2RhaWx5IDwtIHJlYWRfY3N2KGhlcmUoImRhdGEiLCAiVW5pdGVkX1N0YXRlc19DT1ZJRC0xOV9DYXNlc19hbmRfRGVhdGhzX2J5X1N0YXRlX292ZXJfVGltZS5jc3YiKSwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2xfdHlwZXMgPSBjb2xzKHN1Ym1pc3Npb25fZGF0ZSA9IGNvbF9kYXRlKGZvcm1hdCA9ICIlbS8lZC8lWSIpKSkKCmhlYWQobmF0aW9uYWxfY2FzZXNfZGFpbHkpCmBgYAoKXCAgCgojIyMgUHJlcGFyaW5nIGRhdGEKCldlJ2xsIGFnZ3JlZ2F0ZSB0aGUgZGFpbHkgY2FzZSBjb3VudHMgaW50byB3ZWVrcywgc3RhcnRpbmcgb24gU3VuZGF5IGFzIGlzIHRoZSBzYW1lIGFzIGFuIGVwaWRlbWlvbG9naWNhbCB3ZWVrIChyZWZlcnJlZCB0byBhcyBNTVdSIHdlZWssIG1vcmUgaW5mbyBbZm91bmQgaGVyZV0oaHR0cHM6Ly9uZGMuc2VydmljZXMuY2RjLmdvdi93cC1jb250ZW50L3VwbG9hZHMvTU1XUl93ZWVrX292ZXJ2aWV3LnBkZikpCmBgYHtyfQojIGNsZWFuIHVwIGNvbHVtbiBuYW1lcwpuYXRpb25hbF9jYXNlc19kYWlseSA8LSBuYXRpb25hbF9jYXNlc19kYWlseSAlPiUKICBjbGVhbl9uYW1lcygpCgojIGFnZ3JlZ2F0ZSBkYWlseSBDT1ZJRCBjYXNlcyBpbnRvIHdlZWtseSBjYXNlIGNvdW50cywgCiMgc3RhcnQgd2VlayBvbiBTdW5kYXkgYWNjb3JkaW5nIHRvIGVwaWRlbWlvbG9naWNhbCB3ZWVrIChNTVdSIHdlZWspIAp3ZWVrIDwtIGFzLkRhdGUoY3V0KG5hdGlvbmFsX2Nhc2VzX2RhaWx5JGRhdGUsICJ3ZWVrIiwgc3RhcnQub24ubW9uZGF5ID0gRkFMU0UpKQpuYXRpb25hbF9jYXNlc193ZWVrbHkgPC0gYWdncmVnYXRlKG5ld19jYXNlcyB+IHdlZWssIG5hdGlvbmFsX2Nhc2VzX2RhaWx5LCBzdW0pCmhlYWQobmF0aW9uYWxfY2FzZXNfd2Vla2x5KQoKIyBpbmNsdWRlIGxhc3QgZGF5IG9mIHdlZWsgYW5kIGZpbHRlciBkYXRlcyBhZnRlciBBdWd1c3QKbmF0aW9uYWxfY2FzZXNfd2Vla2x5IDwtIG5hdGlvbmFsX2Nhc2VzX3dlZWtseSAlPiUKICBtdXRhdGUod2Vla19lbmQgPSB3ZWVrICsgZGF5cyg2KSkgJT4lCiAgZmlsdGVyKHdlZWsgPCAiMjAyMS0wOS0wMSIpCiAgCmBgYAoKClRoZSBmb3JlY2FzdCBtb2RlbHMgYXJlIHB1bGxlZCBmcm9tIHRoZSBbWm9sdGFyIGZvcmVjYXN0IG1vZGVsIGFyY2hpdmVdKGh0dHBzOi8vd3d3LnpvbHRhcmRhdGEuY29tLykgdXNpbmcgdGhlIFtjb3ZpZEh1YlV0aWxzIHBhY2thZ2VdKGh0dHA6Ly9yZWljaGxhYi5pby9jb3ZpZEh1YlV0aWxzL2luZGV4Lmh0bWwpLiAKCmBgYHtyLCBtZXNzYWdlPUZBTFNFfQojIGNoZWNrIHdoaWNoIFVTLXNwZWNpZmljIG1vZGVscyBhcmUgY29udGFpbmVkIGluIHRoZSBmb3JlY2FzdCBhcmNoaXZlLgptb2RlbF9uYW1lcyA8LSBnZXRfYWxsX21vZGVscyhodWIgPSAiVVMiKSAKYGBgCgpUaGVyZSBhcmUgYHIgbGVuZ3RoKG1vZGVsX25hbWVzKWAgbW9kZWxzIHdpdGhpbiB0aGUgWm9sdGFyIGZvcmVjYXN0IGFyY2hpdmUgZnJvbSB0aGUgVS5TIENPVklELTE5IEZvcmVjYXN0cyBIdWIuCgpMb2FkIHRoZSBpbmNpZGVudCBjYXNlcyBmb3JlY2FzdHMgbW9kZWxzIG9mIDEgdGhyb3VnaCA0IHdlZWsgaG9yaXpvbnMgb2YgYSBzZWxlY3QgbnVtYmVyIG9mIFVTLXNwZWNpZmljIG1vZGVscyB0aGF0IGFyZSBwdWJsaXNoZWQgYnkgdGhlIENEQyBhbmQgYXJlIGNvbnRhaW5lZCB3aXRoaW4gdGhlIFpvbHRhciBmb3JlY2FzdCBhcmNoaXZlLgoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CiMgYnlwYXNzIHF1ZXJ5aW5nIGFsbCBmb3JlY2FzdCBtb2RlbHMgZm9ybSB6b2x0YXIKYnlwYXNzX2xvYWRfZnJvbV96b2x0YXIgPC0gMQpgYGAKCmBgYHtyLCBtZXNzYWdlPUZBTFNFLCBjYWNoZT1UUlVFfQppZiAoYnlwYXNzX2xvYWRfZnJvbV96b2x0YXIgPT0gMCkgewojIGxvYWQgZm9yZWNhc3RzIG9mIGFsbCBtb2RlbHMgIApzeXN0ZW0udGltZShhbGxfaW5jX2Nhc2VfdGFyZ2V0cyA8LSBsb2FkX2ZvcmVjYXN0cygKICBsb2NhdGlvbiA9ICJVUyIsCiAgaHViID0gIlVTIiwKICB0eXBlcyA9IGMoInBvaW50IiwicXVhbnRpbGUiKSwKICB0YXJnZXRzID0gIHBhc3RlKDE6NCwgIndrIGFoZWFkIGluYyBjYXNlIiksCiAgYXNfb2YgPSAiMjAyMS0wOS0wNiIsCiAgc291cmNlID0gInpvbHRhciIpKQogIAojIHdyaXRlIG1vZGVsIGZvcmVjYXN0IHRvIGRpcmVjdG9yeQp3cml0ZV9jc3YoYWxsX2luY19jYXNlX3RhcmdldHMsIGhlcmUoImRhdGEiLCAiYWxsX2luY19jYXNlX3RhcmdldHMuY3N2IikpCn0gZWxzZSB7CiAgIyByZWFkIHByZS1xdWVyaWVkIGZvcmVjYXN0IGRhdGEKICBhbGxfaW5jX2Nhc2VfdGFyZ2V0cyA8LSByZWFkX2NzdihoZXJlKCJkYXRhIiwgImFsbF9pbmNfY2FzZV90YXJnZXRzLmNzdiIpKQp9CgpgYGAKCmBgYHtyLCBtZXNzYWdlPUZBTFNFLCBjYWNoZT1UUlVFfQojIGZpbHRlciBmb3IgIGEgc2VsZWN0IGZldyBwb2ludCBmb3JlY2FzdHMgdGhhdCB3ZXJlIHN1Ym1pdHRlZCBpbiBmcm9tIHpvbHRhciBmb3JlY2FzdCBhcmNoaXZlCiAgaW5jX2Nhc2VfdGFyZ2V0cyA8LSBhbGxfaW5jX2Nhc2VfdGFyZ2V0cyAlPiUgCiAgZmlsdGVyKG1vZGVsICVpbiUgYygiQ09WSURodWItZW5zZW1ibGUiLCAiQ292aWRBbmFseXRpY3MtREVMUEhJIiwgIkNVLXNlbGVjdCIsCiAgICAgICAgICAgICAiSUhNRS1DdXJ2ZUZpdCIsICJMQU5MLUdyb3d0aFJhdGUiLCJVU0MtU0lfa0phbHBoYSIsIAogICAgICAgICAgICAgIkpIVV9JREQtQ292aWRTUCIsICJVVkEtRW5zZW1ibGUiKSkKCmBgYAoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CiMgdGhlIG51bWJlciBvZiBtb2RlbHMgd2l0aCB3ZWVrIGluY2lkZW50IGNhc2UgZm9yZWNhc3RzCiAgbGVuZ3RoKHVuaXF1ZShhbGxfaW5jX2Nhc2VfdGFyZ2V0cyRtb2RlbCkpCgogIGxlbmd0aCh1bmlxdWUoaW5jX2Nhc2VfdGFyZ2V0cyRtb2RlbCkpCmBgYAoKNTIgbW9kZWxzIHN0b3JlZCBpbiB0aGUgWm9sdGFyIGZvcmVjYXN0IGFyY2hpdmUgaGF2ZSBzdWJtaXR0ZWQgYSB3ZWVrbHkgaW5jaWRlbnQgY2FzZSBmb3JlY2FzdC4KCmBgYHtyfQojIGRpc3BsYXkgdG9wIHJvd3Mgb2YgZm9yZWNhc3QgZGF0YQpoZWFkKGluY19jYXNlX3RhcmdldHMpCgppbmNfY2FzZV90YXJnZXRzIDwtIGluY19jYXNlX3RhcmdldHMgJT4lCiAgbXV0YXRlKGZvcmVjYXN0X2RheSA9IGx1YnJpZGF0ZTo6d2RheShmb3JlY2FzdF9kYXRlLCBsYWJlbCA9IFRSVUUsIGFiYnIgPSBGQUxTRSkpICU+JQogIHNlbGVjdChtb2RlbCwgZm9yZWNhc3RfZGF0ZSwgZm9yZWNhc3RfZGF5LCBldmVyeXRoaW5nKCkpCmBgYAoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CiMgYWRqdXN0IGZvcmVjYXN0IGRhdGVzIHRvIGxpZSBvbiB0aGUgc2FtZSB3ZWVrIG9mIHRoZWlyIHJlc3BlY3RpdmUgZm9yZWNhc3QgdGFyZ2V0IGVuZCBkYXRlCiMgaHR0cHM6Ly9naXRodWIuY29tL3JlaWNobGFiL2NvdmlkMTktZm9yZWNhc3QtaHViL2Jsb2IvbWFzdGVyL2RhdGEtcHJvY2Vzc2VkL1JFQURNRS5tZCNmb3JlY2FzdC1maWxlLWZvcm1hdAphZGpfaW5jX2Nhc2VfdGFyZ2V0cyA8LSBpbmNfY2FzZV90YXJnZXRzICU+JQogIG11dGF0ZShhZGpfZm9yZWNhc3RfZGF0ZSA9IGFzLkRhdGUoaWZlbHNlKGZvcmVjYXN0X2RheSA9PSAiU3VuZGF5IiwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvcmVjYXN0X2RhdGUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNhc2Vfd2hlbihmb3JlY2FzdF9kYXkgPT0gIk1vbmRheSIgfiBmb3JlY2FzdF9kYXRlIC0gMSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvcmVjYXN0X2RheSA9PSAiVHVlc2RheSIgfiBmb3JlY2FzdF9kYXRlICsgNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvcmVjYXN0X2RheSA9PSAiV2VkbmVzZGF5IiB+IGZvcmVjYXN0X2RhdGUgKyA0LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9yZWNhc3RfZGF5ID09ICJUaHVyc2RheSIgfiBmb3JlY2FzdF9kYXRlICsgMywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvcmVjYXN0X2RheSA9PSAiRnJpZGF5IiB+IGZvcmVjYXN0X2RhdGUgKyAyLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9yZWNhc3RfZGF5ID09ICJTYXR1cmRheSIgfiBmb3JlY2FzdF9kYXRlICsgMSkpLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb3JpZ2luID0gIjE5NzAtMDEtMDEiKSwKICAgICAgICAgYWRqX2ZvcmVjYXN0X2RheSA9IGx1YnJpZGF0ZTo6d2RheShhZGpfZm9yZWNhc3RfZGF0ZSwgbGFiZWwgPSBUUlVFLCBhYmJyID0gRkFMU0UpKSAlPiUKICBzZWxlY3QoZm9yZWNhc3RfZGF0ZSwgZm9yZWNhc3RfZGF5LCBhZGpfZm9yZWNhc3RfZGF0ZSwgYWRqX2ZvcmVjYXN0X2RheSwgdGFyZ2V0X2VuZF9kYXRlLCBldmVyeXRoaW5nKCkpICU+JQogIGFycmFuZ2UoYWRqX2ZvcmVjYXN0X2RhdGUpICU+JQogICMgZHBseXI6OmRpc3RpbmN0KG1vZGVsLCBhZGpfZm9yZWNhc3RfZGF0ZSwgLmtlZXBfYWxsID0gVFJVRSkgJT4lCiAgZmlsdGVyKGFkal9mb3JlY2FzdF9kYXRlIDwgIjIwMjEtMDktMDEiKQpgYGAKCkZpbHRlciB0aGUgbW9kZWxzIGZvciByZXNwZWN0aXZlIDEgdGhyb3VnaCA0IHdlZWsgaG9yaXpvbiBwb2ludCBmb3JlY2FzdHMuCmBgYHtyfQojIGZpbHRlciBmb3IgMS00IHdlZWsgaG9yaXpvbiBwb2ludCBmb3JlY2FzdHMKZm9yICh3ayBpbiAxOjQpIHsKICB3a19mb3JlY2FzdHMgPC0gYWRqX2luY19jYXNlX3RhcmdldHMgJT4lIAogIGZpbHRlcihob3Jpem9uID09IHdrLAogICAgICAgICB0eXBlID09ICJwb2ludCIpICU+JQogIHNlbGVjdCgtcXVhbnRpbGUpICU+JSAKICBkcGx5cjo6ZGlzdGluY3QobW9kZWwsIGFkal9mb3JlY2FzdF9kYXRlLCAua2VlcF9hbGwgPSBUUlVFKQogIGFzc2lnbihwYXN0ZTAoImluY19jYXNlX3RhcmdldHNfIix3aywiX3dlZWsiKSwgd2tfZm9yZWNhc3RzKQp9CgpgYGAKCiMjIyBWaXN1YWxpemluZyBmb3JlY2FzdCBkYXRhCgpQbG90IHRoZSBDREMgYWN0dWFsIENPVklELTE5IGNhc2VzIHZlcnN1cyB0aGUgZm9yZWNhc3QgcHJlZGljdGlvbnMgZm9yIHRoZSBzZWxlY3QgbW9kZWxzLgpgYGB7cn0KIyBwbG90IHRoZSB3ZWVrbHkgY2FzZXMgZm9yIFVuaXRlZCBTdGF0ZXMgYWNjb3JkaW5nIHRvIENEQyBkYXRhIGFzIG9mIDA5LzA2LzIwMjEKcCA8LSBnZ3Bsb3QoZGF0YSA9IG5hdGlvbmFsX2Nhc2VzX3dlZWtseSwgYWVzKHggPSB3ZWVrX2VuZCwgeSA9IG5ld19jYXNlcykpICsKICBnZW9tX3BvaW50KCkgKwogIGdlb21fbGluZSgpICsKICBsYWJzKHRpdGxlID0gIkNEQyB3ZWVrbHkgaW5jaWRlbnQgQ09WSUQtMTkgY2FzZXMiKQpwCmBgYAoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CiMgcGxvdCB2YXJpb3VzIG1vZGVsIDQgd2VlayBob3Jpem9uIGZvcmVjYXN0cwpnZ3Bsb3QoZGF0YSA9IGluY19jYXNlX3RhcmdldHNfNF93ZWVrLCBhZXMoeCA9IHRhcmdldF9lbmRfZGF0ZSwgeSA9IHZhbHVlLCBjb2xvciA9IG1vZGVsKSkgKwogIGdlb21fcG9pbnQoKSArCiAgZ2VvbV9saW5lKCkgKwogIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKQpgYGAKCmBgYHtyfQojIGNvbWJpbmUgcGxvdHMKcCArIGdlb21fbGluZShkYXRhID0gaW5jX2Nhc2VfdGFyZ2V0c180X3dlZWssIGFlcyh4ID0gdGFyZ2V0X2VuZF9kYXRlLCB5ID0gdmFsdWUsIGNvbG9yID0gbW9kZWwpKSArCiAgZ2VvbV9wb2ludChkYXRhID0gaW5jX2Nhc2VfdGFyZ2V0c180X3dlZWssIGFlcyh4ID0gdGFyZ2V0X2VuZF9kYXRlLCB5ID0gdmFsdWUsIGNvbG9yID0gbW9kZWwpKSArCiAgbGFicyh0aXRsZSA9ICJDREMgd2Vla2x5IGluY2lkZW50IENPVklELTE5IGNhc2VzIHZlcnN1cyB2YXJpb3VzIG1vZGVsIHBvaW50IGZvcmVjYXN0cyIpICsKICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKYGBgCgpJbnRlcmFjdGl2ZSBwbG90IG9mIENEQyBvYnNlcnZlZCBjYXNlcyB2ZXJzdXMgNCB3ZWVrIGhvcml6b24gZm9yZWNhc3RzCmBgYHtyLCBpbmNsdWRlPUZBTFNFfQpwbG90X2luY19mb3JlY2FzdCA8LSAgZnVuY3Rpb24oaG9yaXpvbiA9IDEpIHsKICBmaWcxIDwtIHBsb3RfbHkodHlwZSA9ICdzY2F0dGVyJywgIG1vZGUgPSAnbGluZXMrbWFya2VycycpCiAgZmlnMSA8LSBmaWcxICU+JSAKICAgIGFkZF90cmFjZShkYXRhID0gZ2V0KChwYXN0ZTAoImluY19jYXNlX3RhcmdldHNfIixob3Jpem9uLCJfd2VlayIpKSksIHggPSB+dGFyZ2V0X2VuZF9kYXRlLCB5ID0gfnZhbHVlLCBjb2xvciA9IH5mYWN0b3IobW9kZWwpKSAlPiUKICAgIGFkZF90cmFjZShkYXRhID0gbmF0aW9uYWxfY2FzZXNfd2Vla2x5LCB4ID0gfndlZWtfZW5kLCB5ID0gfm5ld19jYXNlcywgbmFtZSA9ICJDREMgYWN0dWFsIiwgY29sb3IgPSBJKCdibGFjaycpKSAlPiUKICAgIGxheW91dCh0aXRsZSA9IHBhc3RlKCJDREMgd2Vla2x5IGluY2lkZW50IENPVklELTE5IGNhc2VzXG4gYW5kIiwgaG9yaXpvbiwgIndlZWsgaG9yaXpvbiBwb2ludCBmb3JlY2FzdHMiKSkKICBmaWcxCn0KCmBgYAoKYGBge3J9CiMgcGxvdCAxIHdlZWsgaG9yaXpvbiBwb2ludCBmb3JlY2FzdApwbG90X2luY19mb3JlY2FzdChob3Jpem9uID0gMSkKCiMgcGxvdCA0IHdlZWsgaG9yaXpvbiBwb2ludCBmb3JlYWNzdApwbG90X2luY19mb3JlY2FzdChob3Jpem9uID0gNCkKYGBgCgpJSE1FLUN1cnZlZml0IG9ubHkgcHJlZGljdGVkIGluY2lkZW50IENPVklELWNhc2VzIGZvciBhIGZldyB3ZWVrcyBiZWZvcmUgZm9jdXNpbmcgb24gcHJlZGljdGluZyBob3NwaXRhbGl6YXRpb25zIGFuZCBkZWF0aHMsIGFjY29yZGluZyB0byBwcmVkaWN0aW9ucyBzdG9yZWQgaW4gWm9sdGFyIGZvcmVjYXN0IGFyY2hpdmUuCgpJZGVudGlmeWluZyBwZWFrcyBvZiBDT1ZJRC0xOSBjYXNlcwpgYGB7cn0KIyBhICdwZWFrJyBpcyBkZWZpbmVkIGFzIGEgbG9jYWwgbWF4aW1hIHdpdGggbSBwb2ludHMgZWl0aGVyIHNpZGUgb2YgaXQgYmVpbmcgc21hbGxlciB0aGFuIGl0LiBoZW5jZSwgdGhlIGJpZ2dlciB0aGUgcGFyYW1ldGVyIG0sIHRoZSBtb3JlIHN0cmluZ2VudCBpcyB0aGUgcGVhayBmaW5kaW5nIHByb2NlZHVyZQojIGh0dHBzOi8vZ2l0aHViLmNvbS9zdGFzLWcvZmluZFBlYWtzCmZpbmRfcGVha3MgPC0gZnVuY3Rpb24gKHgsIG0gPSAzKXsKICAgIHNoYXBlIDwtIGRpZmYoc2lnbihkaWZmKHgsIG5hLnBhZCA9IEZBTFNFKSkpCiAgICBwa3MgPC0gc2FwcGx5KHdoaWNoKHNoYXBlIDwgMCksIEZVTiA9IGZ1bmN0aW9uKGkpewogICAgICAgeiA8LSBpIC0gbSArIDEKICAgICAgIHogPC0gaWZlbHNlKHogPiAwLCB6LCAxKQogICAgICAgdyA8LSBpICsgbSArIDEKICAgICAgIHcgPC0gaWZlbHNlKHcgPCBsZW5ndGgoeCksIHcsIGxlbmd0aCh4KSkKICAgICAgIGlmKGFsbCh4W2MoeiA6IGksIChpICsgMikgOiB3KV0gPD0geFtpICsgMV0pKSByZXR1cm4oaSArIDEpIGVsc2UgcmV0dXJuKG51bWVyaWMoMCkpCiAgICB9KQogICAgIHBrcyA8LSB1bmxpc3QocGtzKQogICAgIHBrcwp9CmBgYAoKClBlYWtzIG9mIG9ic2VydmVkIHdlZWtseSBpbmNpZGVudCBDT1ZJRC0xOSBjYXNlcyBiYXNlZCBvbiBDREMgZGF0YQpgYGB7cn0KIyBmaW5kIHBlYWtzIG9mIG9ic2VydmVkIGNhc2VzCnBlYWtfY2FzZXMgPC0gbmF0aW9uYWxfY2FzZXNfd2Vla2x5W2ZpbmRfcGVha3MobmF0aW9uYWxfY2FzZXNfd2Vla2x5JG5ld19jYXNlcywgbSA9IDMpLF0KCiMgcGxvdCBwZWFrcyBmb3Igb2JzZXJ2ZWQgY292aWQgY2FzZXMKcSA8LSBwICsgZ2VvbV9wb2ludChkYXRhID0gcGVha19jYXNlcywgYWVzKHggPSB3ZWVrX2VuZCwgeSA9IG5ld19jYXNlcywgY29sb3IgPSAiQ0RDIGFjdHVhbCIpKSArCiAgbGFicyh0aXRsZSA9ICJPYnNlcnZlZCBwZWFrcyBvZiB3ZWVrbHkgaW5jaWRlbnQgQ09WSUQtMTkgY2FzZXMiLCAKICAgICAgIHggPSAiV2Vla3MiLCB5ID0gIkluY2lkZW50IGNhc2VzIiwgY29sb3IgPSAiUGVha3MiKSArCiAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCgpxCmBgYAoKYGBge3J9CiMgZ2V0IHBlYWsgdmFsdWVzIG9mIGFsbCB2YWx1ZXMKbW9kZWxfcGVha3NfbGlzdCA8LSBsaXN0KCkKCiMgZ2V0IHBlYWtzIGZvciA0IHdlZWsgaG9yaXpvbiBmb3JlY2FzdApmb3IgKGkgaW4gdW5pcXVlKGluY19jYXNlX3RhcmdldHNfNF93ZWVrJG1vZGVsKSkgewogIG1vZGVscyA8LSBpbmNfY2FzZV90YXJnZXRzXzRfd2VlayAlPiUKICAgIGZpbHRlcihtb2RlbCA9PSBpKQogIHBlYWtzIDwtIG1vZGVsc1tmaW5kX3BlYWtzKG1vZGVscyR2YWx1ZSwgbSA9IDMpLF0KICBtb2RlbF9wZWFrc19saXN0W1tpXV0gPC0gcGVha3MKfQoKIyBjb21iaW5lIHRoZSBkZiBvZiBwZWFrcwptb2RlbF9wZWFrcyA8LSBiaW5kX3Jvd3MobW9kZWxfcGVha3NfbGlzdCkKYGBgCgpMb29rIGF0IHRoZSBwZWFrcyBhc3NvY2lhdGVkIHdpdGggZm9yZWNhc3QgbW9kZWxzCmBgYHtyfQojIGNvbWJpbmUgcGxvdHMgb2YgY2RjIGFjdHVhbCBwZWFrcyB3aXRoIGZvcmVjYXN0IG1vZGVsIHBlYWtzCnEgKyBnZW9tX3BvaW50KGRhdGEgPSBtb2RlbF9wZWFrcywgYWVzKHggPSB0YXJnZXRfZW5kX2RhdGUsIHkgPSB2YWx1ZSwgY29sb3IgPSBtb2RlbCkpCmBgYAoKYGBge3J9CiMgZmluZCBtYWduaXR1ZGUgYW5kIHRlbXBvcmFsIGRpZmZlcmVuY2VzIGluIHRoZSBwZWFrcwpjZGNfcGVha3MgPC0gcGVha19jYXNlcyAlPiUKICBtdXRhdGUobW9kZWwgPSByZXAoIkNEQyIsIG5yb3cocGVha19jYXNlcykpKSAlPiUKICBzZWxlY3Qod2Vla19lbmQsIG1vZGVsLCBuZXdfY2FzZXMpICU+JQogIHJlbmFtZSgicGVha3MiID0gIm5ld19jYXNlcyIpCgptb2RlbF9vYnNlcnZlZF9wa3MgPC0gbW9kZWxfcGVha3MgJT4lCiAgc2VsZWN0KHRhcmdldF9lbmRfZGF0ZSwgbW9kZWwsIHZhbHVlKSAlPiUKICByZW5hbWUoIndlZWtfZW5kIiA9ICJ0YXJnZXRfZW5kX2RhdGUiLAogICAgICAgICAicGVha3MiID0gInZhbHVlIikgJT4lCiAgYmluZF9yb3dzKGNkY19wZWFrcykKCmBgYAoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CiMgIyBmaW5kIGVhcmxpZXN0IHBlYWsgZm9yIGVhY2ggb2YgdGhlIG1vZGVscwojIG1vZGVsX29ic2VydmVkX3BrcyAlPiUKIyAgIGdyb3VwX2J5KG1vZGVsKSAlPiUKIyAgIHN1bW1hcmlzZShmaXJzdF9wa19kYXRlID0gbWluKHdlZWtfZW5kKSkKYGBgCgojIyMgU2NvcmUgZWFjaCBtb2RlbCB3ZWVrbHkKVG8gc2NvcmUgbW9kZWxzIHVzaW5nIGNvdmlkSHViVXRpbHMsIGRhdGEgZnJhbWUgbmVlZHMgdG8gYmUgaW4gc2FtZSBmb3JtYXQgYXMgb25lIGdvdHRlbiBmcm9tIGBsb2FkX3RydXRoYCBmdW5jdGlvbi4KYGBge3IsIG1lc3NhZ2U9RkFMU0UsIGNhY2hlPVRSVUV9CiMgbG9hZCB0cnV0aCBkYXRhIGZyYW1lIGZyb20gZmlsZQpieXBhc3NfdHJ1dGhfem9sdGFyIDwtIDEKCiMgbG9hZCBhIHRydXRoIGRhdGEgZnJhbWUgZnJvbSBjb3ZpZEh1YgppZiAoYnlwYXNzX3RydXRoX3pvbHRhciA9PSAwKSB7CiAgamh1X3RydXRoX2RmIDwtIGxvYWRfdHJ1dGgoCiAgICB0cnV0aF9zb3VyY2UgPSBjKCJKSFUiKSwKICAgIHRhcmdldF92YXJpYWJsZSA9IGMoImluYyBjYXNlIiksCiAgICB0cnV0aF9lbmRfZGF0ZSA9ICIyMDIxLTA5LTA0IiwKICAgIHRlbXBvcmFsX3Jlc29sdXRpb24gPSAid2Vla2x5IiwKICAgIGh1YiA9ICJVUyIsCiAgICBsb2NhdGlvbnMgPSAgIlVTIikKICAKICAjIHdyaXRlIHRydXRoIGRhdGEgZnJhbWUgdG8gY3N2IGZpbGUKICB3cml0ZV9jc3Yoamh1X3RydXRoX2RmLCBoZXJlKCJkYXRhIiwgImpodV90cnV0aF9kZi5jc3YiKSkKICB9IGVsc2UgewogICAgCiAgICAjIGxvYWQgdHJ1dGggZGYgZnJvbSBmaWxlCiAgICBqaHVfdHJ1dGhfZGYgPC0gcmVhZF9jc3YoaGVyZSgiZGF0YSIsICJqaHVfdHJ1dGhfZGYuY3N2IikpCiAgICB9CgojIHRvcCByb3dzIG9mIHRydXRoIGRhdGFmcmFtZQpoZWFkKGpodV90cnV0aF9kZikKYGBgCgpgYGB7ciwgaW5jbHVkZT1GQUxTRX0KIyB0byBzY29yZSBtb2RlbHMsIGRhdGEgZnJhbWUgbmVlZHMgdG8gYmUgaW4gc2FtZSBmb3JtYXQgYXMgb25lIGdvdHRlbiBmcm9tIGxvYWRfdHJ1dGgKIyBUT0RPOiBjb252ZXJ0IG5hdGlvbmFsX2Nhc2VzX3dlZWtseSB0byBqaHVfdHJ1dGhfZGYgZm9ybWF0CgojIGZpbHRlciBkYXRlcyBhZnRlciBTZXB0ZW1iZXIgNiwgMjAyMQpqaHVfdHJ1dGhfZGYyIDwtIGpodV90cnV0aF9kZiAlPiUgCiAgZmlsdGVyKHRhcmdldF9lbmRfZGF0ZSA8ICIyMDIxLTA5LTA2IikKCiMgam9pbiBDREMgb2JzZXJ2ZWQgY2FzZXMgYW5kIHJlYXJyYW5nZSB0cnV0aCBkYXRhZnJhbWUKamh1X3RydXRoX2RmMiA8LSBqaHVfdHJ1dGhfZGYyICU+JQogIGZ1bGxfam9pbihuYXRpb25hbF9jYXNlc193ZWVrbHksIGJ5ID0gYygidGFyZ2V0X2VuZF9kYXRlIiA9ICJ3ZWVrX2VuZCIpKSAlPiUKICBzZWxlY3Qoamh1X21vZGVsID0gbW9kZWwsIGpodV92YWx1ZSA9IHZhbHVlLCB2YWx1ZSA9IG5ld19jYXNlcywgZXZlcnl0aGluZygpKQoKIyBhZGQgbmFtZSBmb3IgQ0RDIG9ic2VydmVkIGRhdGEgYW5kIHJlYXJyYW5nZQp0cnV0aF9kZiA8LSBqaHVfdHJ1dGhfZGYyICU+JQogIG11dGF0ZShtb2RlbCA9IHJlcCgiT2JzZXJ2ZWQgZGF0YSAoQ0RDKSIsIG5yb3coamh1X3RydXRoX2RmMikpKSAlPiUKICBzZWxlY3QobW9kZWwsIGV2ZXJ5dGhpbmcoKSkgJT4lCiAgc2VsZWN0KC1qaHVfbW9kZWwsIC1qaHVfdmFsdWUpCgojdXNlIHNjb3JlX2ZvcmVjYXN0cyBmdW5jdGlvbiB0byBjb21wdXRlIHdlaWdodGVkIGludGVydmFsIHNjb3JlIGFuZCBvdGhlciBtZXRyaWNzCmZvcmVjYXN0X21vZGVsX3Njb3JlcyA8LSBzY29yZV9mb3JlY2FzdHMoZm9yZWNhc3RzID0gYWxsX2luY19jYXNlX3RhcmdldHMsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cnV0aCA9IHRydXRoX2RmLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWV0cmljcyA9IGMoImFic19lcnJvciIsICJ3aXMiLCAid2lzX2NvbXBvbmVudHMiLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImludGVydmFsX2NvdmVyYWdlIikpCgpgYGAKCgpgYGB7ciwgaW5jbHVkZT1GQUxTRX0KIyBqb2luIGZvcmVjYXN0IGRmIHRvIGdldCBmb3JlY2FzdCB2YWx1ZXMgZm9yIGVhY2ggbW9kZWwKam9pbnRfZGYgPC0gYWxsX2luY19jYXNlX3RhcmdldHMgJT4lCiAgZmlsdGVyKHR5cGUgPT0gInBvaW50IikgJT4lCiAgcmlnaHRfam9pbihmb3JlY2FzdF9tb2RlbF9zY29yZXMsIGJ5ID0gYygibW9kZWwiLCAiaG9yaXpvbiIsImZvcmVjYXN0X2RhdGUiLCJ0YXJnZXRfZW5kX2RhdGUiKSkgJT4lIAogIGRwbHlyOjpyZW5hbWUoImZvcmVjYXN0X3ZhbHVlIiA9IHZhbHVlKQpgYGAKCldJUyBzY29yZXMgZm9yIGVhY2ggbW9kZWwgYW5kIGRpZmZlcmVudCBmb3JlY2FzdCBob3Jpem9ucwpgYGB7cn0KIyBnZXQgZWFjaCBtb2RlbHMgd2Vla2x5IHdpcyBzY29yZSBmb3IgZWFjaCBob3Jpem9uCmZvciAoaSBpbiAxOjQpIHsKICB3aXNfc2NvcmVzIDwtIGpvaW50X2RmICU+JSAKICAgIGZpbHRlcihob3Jpem9uID09IGkpICU+JQogICAgIyBncm91cF9ieShtb2RlbCwgdGFyZ2V0X2VuZF9kYXRlKSAlPiUKICAgIHNlbGVjdChtb2RlbCwgaG9yaXpvbiwgYWJzX2Vycm9yLCB3aXMsIGZvcmVjYXN0X3ZhbHVlLCB0cnVlX3ZhbHVlLCB0YXJnZXRfZW5kX2RhdGUsIGZvcmVjYXN0X2RhdGUpICU+JQogICAgYXJyYW5nZShkZXNjKHdpcykpCiAgCiAgYXNzaWduKHBhc3RlMCgid2lzX3Njb3Jlc18iLGkpLCB3aXNfc2NvcmVzKQp9Cndpc19zY29yZXNfMQp3aXNfc2NvcmVzXzIKd2lzX3Njb3Jlc18zCndpc19zY29yZXNfNApgYGAKCk1lZGlhbiBhYnNvbHV0ZSBwZXJjZW50IGVycm9yIChtYXBlKSBmb3IgZWFjaCBtb2RlbCBhbmQgZm9yZWNhc3QgaG9yaXpvbgpgYGB7ciwgbWVzc2FnZT1GQUxTRX0KIyBjYWxjdWxhdGUgdGhlIG1lZGlhbiBhYnNvbHV0ZSBwZXJjZW50IGVycm9yIGZvciBlYWNoIG1vZGVsCmpvaW50X2RmICU+JQogIGdyb3VwX2J5KG1vZGVsLCBob3Jpem9uKSAlPiUKICBzdW1tYXJpc2UobnVtX29mX2ZvcmVjYXN0cyA9IG4oKSwKICAgICAgICAgICAgbWFwZSA9IG1lZGlhbihhYnMoKHRydWVfdmFsdWUtZm9yZWNhc3RfdmFsdWUpL3RydWVfdmFsdWUpKSoxMDApICU+JQogIGFycmFuZ2UoZGVzYyhtYXBlKSkKYGBgCgpgYGB7cn0KIyBmaWx0ZXIgb3V0IGRhdGVzIGJlZm9yZSBKdWx5IDIwMjAsIGxhYm9yIGRheSwgdGhhbmtzZ2l2aW5nLCBDaHJpc3RtYXMgYW5kIE5ldyBZZWFycwpuYXRpb25hbF9jYXNlc193ZWVrbHkgJT4lCiAgZmlsdGVyKHdlZWtfZW5kID4gIjIwMjAtMDctMDQiICYgCiAgICAgICAgICAgIXdlZWtfZW5kID09ICIyMDIwLTA5LTEyIiAmIAogICAgICAgICAgICF3ZWVrX2VuZCA9PSAiMjAyMC0xMi0yNiIgJiAKICAgICAgICAgICAhd2Vla19lbmQgPT0gIjIwMjEtMDEtMDIiKQogIAoKCiMgY2hlY2sgdGhlIHdlZWtzIG9mIG1vbm90b25pYyBpbmNyZWFzaW5nIGFuZCBkZWNyZWFzaW5nIGNhc2VzIHRpbWUgcGVyaW9kcwp3ZWVrbHlfY2FzZXMgPC0gZGF0YS50YWJsZShuYXRpb25hbF9jYXNlc193ZWVrbHkpCndlZWtseV9jYXNlcyA8LSB3ZWVrbHlfY2FzZXMgJT4lCiAgbXV0YXRlKGRpZmYgPSBuZXdfY2FzZXMgLSBsYWcobmV3X2Nhc2VzLCAxLCAwKSwKICAgICAgICAgaW5jID0gaWZfZWxzZShkaWZmPjAsIDEsIDApLAogICAgICAgICBkZWMgPSBpZl9lbHNlKGRpZmY8MCwgMSwgMCksCiAgICAgICAgIGluY19kZWNfY291bnQgPSBzZXF1ZW5jZShybGUoYXMuY2hhcmFjdGVyKGluYykpJGxlbmd0aHMpKQoKIyBjb25zZWN1dGl2ZSB3ZWVrcyBvZiBpbmNyZWFzaW5nIENPVklEIGNhc2VzIAp3ZWVrbHlfY2FzZXNbd2hpY2goaW5jPT0xKSwgXQoKIyBjb25zZWN1dGl2ZSB3ZWVrcyBvZiBpbmNyZWFzaW5nIENPVklEIGNhc2VzIAp3ZWVrbHlfY2FzZXNbd2hpY2goaW5jPT0wKSwgXQoKCiMgY2hlY2sgdGhlIHdlZWtzIG9mIG1vbm90b25pYyBpbmNyZWFzaW5nIGFuZCBkZWNyZWFzaW5nIGNhc2VzIHRpbWUgcGVyaW9kcyBmb3IgZWFjaCBtb2RlbAptb2RlbHNfaW5jX2RlYyA8LSBhbGxfaW5jX2Nhc2VfdGFyZ2V0cyAlPiUKICBmaWx0ZXIodHlwZSA9PSAicG9pbnQiLAogICAgICAgICBob3Jpem9uID09IDQpICU+JQogIHNlbGVjdCgtbG9jYXRpb24sIC10ZW1wb3JhbF9yZXNvbHV0aW9uLCAtdGFyZ2V0X3ZhcmlhYmxlLCAtdHlwZSwgLXF1YW50aWxlLCAtbG9jYXRpb25fbmFtZSwgCiAgICAgICAgIC1wb3B1bGF0aW9uLCAtc3RhcnRzX3dpdGgoImdlbyIpLCAtYWJicmV2aWF0aW9uLCAtZnVsbF9sb2NhdGlvbl9uYW1lKSAlPiUKICBncm91cF9ieShtb2RlbCkgJT4lCiAgbXV0YXRlKGRpZmYgPSB2YWx1ZSAtIGxhZyh2YWx1ZSwgMSwwKSwKICAgICAgICAgaW5jID0gaWZfZWxzZShkaWZmPjAsIDEsIDApLAogICAgICAgICBkZWMgPSBpZl9lbHNlKGRpZmY8MCwgMSwgMCksCiAgICAgICAgIGluY19kZWNfY291bnQgPSBzZXF1ZW5jZShybGUoYXMuY2hhcmFjdGVyKGluYykpJGxlbmd0aHMpKQoKbW9kZWxzX2luY19kZWMKYGBgCgpgYGB7cn0KIyBjaGVjayB0aGUgd2Vla3Mgb2YgbW9ub3RvbmljIGluY3JlYXNpbmcgYW5kIGRlY3JlYXNpbmcgY2FzZXMgdGltZSBwZXJpb2RzIGZvciBlYWNoIG1vZGVsCm1vZGVsc19ob3Jpem9uX2luY19kZWMgPC0gYWxsX2luY19jYXNlX3RhcmdldHMgJT4lCiAgZmlsdGVyKHR5cGUgPT0gInBvaW50IikgJT4lCiAgc2VsZWN0KC1sb2NhdGlvbiwgLXRlbXBvcmFsX3Jlc29sdXRpb24sIC10YXJnZXRfdmFyaWFibGUsIC10eXBlLCAtcXVhbnRpbGUsIC1sb2NhdGlvbl9uYW1lLCAKICAgICAgICAgLXBvcHVsYXRpb24sIC1zdGFydHNfd2l0aCgiZ2VvIiksIC1hYmJyZXZpYXRpb24sIC1mdWxsX2xvY2F0aW9uX25hbWUpICU+JQogIGdyb3VwX2J5KG1vZGVsLCBob3Jpem9uKSAlPiUKICBtdXRhdGUoZGlmZiA9IHZhbHVlIC0gbGFnKHZhbHVlLCAxLDApLAogICAgICAgICBpbmMgPSBpZl9lbHNlKGRpZmY+MCwgMSwgMCksCiAgICAgICAgIGRlYyA9IGlmX2Vsc2UoZGlmZjwwLCAxLCAwKSwKICAgICAgICAgaW5jX2RlY19jb3VudCA9IHNlcXVlbmNlKHJsZShhcy5jaGFyYWN0ZXIoaW5jKSkkbGVuZ3RocykpCgptb2RlbHNfaG9yaXpvbl9pbmNfZGVjCmBgYAoKCgoKCg==